#!/usr/bin/perl

use DBI;
use strict;
use POSIX;

# Some setup variables
my $rpfits_summary_command="rpfits_summary";
my $rpfits_extract_command="/n/ste616/bin/rpfex";
my $staging_area="/data/MENTOK_2/ste616/flux_monitoring/staging_area";

my $dsn="dbi:Pg:dbname=flux_monitoring;host=localhost;port=5432";
my $user="postgres";
my $password="";

# Set up a connection to the database listing the RPFITS files that need to be
# processed.
my $dbh=DBI->connect($dsn,$user,$password,{ RaiseError => 0, AutoCommit => 1});

# Query for all RPFITS files that need processing.
my $sth=$dbh->prepare("SELECT * FROM rpfits_files WHERE processed=FALSE AND nodata=FALSE ORDER BY file LIMIT 50");
$sth->execute;
my @files_to_process;
my @just_files;
while (my ($rpfits_file,$rpfits_path,$tp)=$sth->fetchrow()){
    print "$rpfits_path/$rpfits_file\n";
    push @files_to_process,"$rpfits_path/$rpfits_file";
    push @just_files,$rpfits_file;
}
$sth->finish;

for (my $i=0;$i<=$#files_to_process;$i++){
    # Get a summary of the FITS file.
    my @header_numbers;
    my @dates;
    my @times;
    my @sources;
    my @if1_freqs;
    my @if2_freqs;
    my @calibrator_types;
    my @projects;
    my @lastpoint_times;
    my @lastpoint_sources;
    my @lastpoint_azs;
    my @lastpoint_els;
    my $command=$rpfits_summary_command;
    $command .= " ".$files_to_process[$i];
    my $this_project=$files_to_process[$i];
    $this_project=~s/(.*)\.(.*)$/$2/;
    print "executing $command\n";
    open(SUMMARY,"-|")||exec $command;
    while(<SUMMARY>){
	chomp;
	my $line=$_;
	print "$line\n";
	my @els=split(/\s+/,$line);
	if ($els[0] eq ">>>"){
	    next;
	}
	my $test_name=$els[3];
	my $header_number=$els[0];
	$header_number=~s/\://;
	# Check whether this header describes an observation of a known calibrator.
	my $ccl=$dbh->prepare("SELECT type FROM calibrators WHERE name='$test_name'");
	$ccl->execute;
	push @header_numbers,$header_number;
	push @dates,$els[1];
	push @times,$els[2];
	push @sources,$test_name;
	push @if1_freqs,$els[4];
	push @if2_freqs,$els[7];
	push @calibrator_types,"";
	push @projects,$this_project;
	push @lastpoint_times,"";
	push @lastpoint_sources,"";
	push @lastpoint_azs,0;
	push @lastpoint_els,0;
	while(my ($calibrator_type)=$ccl->fetchrow()){
	    print "$test_name is a $calibrator_type calibrator\n";
	    $calibrator_types[$#calibrator_types]=$calibrator_type;
	}
	$ccl->finish;
    }
    close(SUMMARY);

    # now we use rpfhdr to get some info about the events that
    # this FITS file describes. events are things like paddles,
    # and calibration commands
    $command="rpfhdr ";
    $command.=$files_to_process[$i];
    open(EVENTS,"-|")||exec $command;
    my $currheader;
    my ($event_type,$event_time,$acal_if1_value,$acal_if2_value,
	$event_source,$if1_freq,$if2_freq,$project,$rpfits_file);
    $rpfits_file=$just_files[$i];
    while(<EVENTS>){
	chomp;
	my $line=$_;
	my @els=split(/\s+/,$line);
	if ($line=~/^RPFITS header number/){
	    $currheader=$els[3];
	} elsif ($line=~/^OBSLOG/){
	    # look for calibration commands
	    # get some info from the header
	    if (($line=~/acal/)||($line=~/pcal/)||($line=~/dcal/)){
		my $was_applied=0;
		for (my $j=0;$j<=$#sources;$j++){
		    if ($header_numbers[$j]==$currheader){
			$event_source=$sources[$j];
			$if1_freq=$if1_freqs[$j];
			$if2_freq=$if2_freqs[$j];
			$project=$projects[$j];
			last;
		    }
		}
		$event_time=$els[1]." ".$els[2];
		if ($line=~/acal/){
		    $event_type="acal";
		    # only continue if there is an a at the end
		    # ie. the calibration was applied
		    if ($els[$#els] eq "a"){
			$was_applied=1;
			if ($els[$#els-1] eq "acal"){
			    # acal was called with no arguments
			    $acal_if1_value=1.0;
			    $acal_if2_value=1.0;
			} elsif ($els[$#els-3] eq "acal"){
			    # acal was called with the correct 2 arguments
			    $acal_if1_value=$els[$#els-2];
			    $acal_if2_value=$els[$#els-1];
			}
		    }
		} elsif ($line=~/pcal/){
		    $event_type="pcal";
		    if (($els[$#els] eq "a")&&($els[$#els-1] eq "pcal")){
			$was_applied=1;
			$acal_if1_value=-1.0;
			$acal_if2_value=-1.0;
		    }
		} elsif ($line=~/dcal/){
		    $event_type="dcal";
		    if (($els[$#els] eq "a")&&($els[$#els-1] eq "dcal")){
			$was_applied=1;
			$acal_if1_value=-1.0;
			$acal_if2_value=-1.0;
		    }
		}
		if ($was_applied==1){
		    # put this info into the events table
		    my $evi_query="INSERT INTO events VALUES('".$event_type."',".
			"'".$event_time."',".$acal_if1_value.",".$acal_if2_value.",'".
			$event_source."',".$if1_freq.",".$if2_freq.",'".$project."','".
			$rpfits_file."')";
		    print "$evi_query\n";
		    my $evi=$dbh->prepare($evi_query);
		    $evi->execute;
		    $evi->finish;
		    $was_applied=0;
		}
	    }
	} elsif ($line=~/^SCANTYPE= PADDLE/){
	    # a paddle event
	    $event_type="paddle";
	    for (my $j=0;$j<=$#sources;$j++){
		if ($header_numbers[$j]==$currheader){
		    $event_source=$sources[$j];
		    $if1_freq=$if1_freqs[$j];
		    $if2_freq=$if2_freqs[$j];
		    $event_time=$dates[$j]." ".&decimal_to_time($times[$j]);
		    $project=$projects[$j];
		    last;
		}
	    }
	    $acal_if1_value=-1.0;
	    $acal_if2_value=-1.0;
	    # put this info into the events table
	    my $evi_query="INSERT INTO events VALUES('".$event_type."',".
		"'".$event_time."',".$acal_if1_value.",".$acal_if2_value.",'".
		$event_source."',".$if1_freq.",".$if2_freq.",'".$project."','".
		$rpfits_file."')";
	    print "$evi_query\n";
	    my $evi=$dbh->prepare($evi_query);
	    $evi->execute;
	    $evi->finish;
	} elsif ($line=~/^POINTINF/){
	    # info about the last pointing
	    for (my $j=0;$j<=$#sources;$j++){
		if ($header_numbers[$j]==$currheader){
		    $lastpoint_times[$j]=$els[1]." ".$els[2];
		    $lastpoint_sources[$j]=$els[4];
		    $lastpoint_azs[$j]=$els[6];
		    $lastpoint_els[$j]=$els[8];
		    last;
		}
	    }
	}
    }
    close(EVENTS);

    for (my $j=0;$j<=$#sources;$j++){
	print "source $j: $sources[$j]\n";
    }

    # Now we have to extract out the data from the RPFITS file.
    for (my $j=0;$j<=$#sources;$j++){
	if ($times[$j]==0.000000){
	    next;
	}
	my $this_source=$sources[$j];
	my $start_header_number=$header_numbers[$j];
	my $end_header_number=$header_numbers[$j];
	for (my $k=$j+1;$k<=$#sources;$k++){
	    if (($sources[$k] eq $this_source)&&
		($if1_freqs[$k]==$if1_freqs[$j])&&
		($if2_freqs[$k]==$if2_freqs[$j])){
		$end_header_number=$header_numbers[$k];
	    } else {
		last;
	    }
	}
	print "source $this_source goes from header $start_header_number to $end_header_number\n";
	if ($calibrator_types[$j] ne ""){
	    my $this_time=&decimal_to_time($times[$j]);
	    my $output_file=$this_source."_".$dates[$j]."_".$this_time."_".$if1_freqs[$j].".fits";
	    my $output_full=$staging_area."/".$output_file;
	    my $extract_command=$rpfits_extract_command;
	    $extract_command .= " -x$start_header_number:$end_header_number";
	    $extract_command .= " -o ".$output_full;
	    $extract_command .= " ".$files_to_process[$i];
	    print "will execute command $extract_command\n";
	    system $extract_command;
	    my $rpfits_only=$files_to_process[$i];
	    $rpfits_only=~s/(.*)\/(.*)$/$2/;
	    my $ief_query="INSERT INTO extracted_files VALUES('$output_file','$staging_area',".
				  "'$projects[$j]','$this_source','$dates[$j] $this_time',".
				  "'$rpfits_only',$start_header_number,$end_header_number,".
				  "$if1_freqs[$j],$if2_freqs[$j],";
	    
	    if ($lastpoint_times[$j] ne ""){
		$ief_query.="'$lastpoint_times[$j]','$lastpoint_sources[$j]',$lastpoint_azs[$j],".
		    "$lastpoint_els[$j])";
	    } else {
		$ief_query.="NULL,NULL,NULL,NULL)";
	    }
	    print "$ief_query\n";
	    my $ief=$dbh->prepare($ief_query);
	    $ief->execute;
	    $ief->finish;
	}
	$j=$end_header_number-1;
    }
    my $urp_query="UPDATE rpfits_files SET processed=TRUE WHERE file='".
	$just_files[$i]."'";
    print "$urp_query\n";
    my $urp=$dbh->prepare($urp_query);
    $urp->execute;
    $urp->finish;
}

$dbh->disconnect;

sub decimal_to_time {
    my $decimal_time=$_[0];
#    print "1: $decimal_time\n";
    # we don't care about fractions of a second
    my $whole_seconds=floor($decimal_time);
    my $minutes=$whole_seconds/60.0;
    my $whole_minutes=floor($minutes);
    my $seconds=$whole_seconds-($whole_minutes*60.0);
    my $hours=$whole_minutes/60.0;
    my $whole_hours=floor($hours);
    my $minutes=$whole_minutes-($whole_hours*60.0);
    my $timestring=sprintf "%02d:%02d:%02d",$whole_hours,$minutes,$seconds;
    $timestring;
}
